static char help[] = "Second Order TVD Finite Volume Example.\n";
/*F

We use a second order TVD finite volume method to evolve a system of PDEs. Our simple upwinded residual evaluation loops
over all mesh faces and uses a Riemann solver to produce the flux given the face geometry and cell values,
\begin{equation}
  f_i = \mathrm{riemann}(\mathrm{phys}, p_\mathrm{centroid}, \hat n, x^L, x^R)
\end{equation}
and then update the cell values given the cell volume.
\begin{eqnarray}
    f^L_i &-=& \frac{f_i}{vol^L} \\
    f^R_i &+=& \frac{f_i}{vol^R}
\end{eqnarray}

As an example, we can consider the shallow water wave equation,
\begin{eqnarray}
     h_t + \nabla\cdot \left( uh                              \right) &=& 0 \\
  (uh)_t + \nabla\cdot \left( u\otimes uh + \frac{g h^2}{2} I \right) &=& 0
\end{eqnarray}
where $h$ is wave height, $u$ is wave velocity, and $g$ is the acceleration due to gravity.

A representative Riemann solver for the shallow water equations is given in the PhysicsRiemann_SW() function,
\begin{eqnarray}
  f^{L,R}_h    &=& uh^{L,R} \cdot \hat n \\
  f^{L,R}_{uh} &=& \frac{f^{L,R}_h}{h^{L,R}} uh^{L,R} + g (h^{L,R})^2 \hat n \\
  c^{L,R}      &=& \sqrt{g h^{L,R}} \\
  s            &=& \max\left( \left|\frac{uh^L \cdot \hat n}{h^L}\right| + c^L, \left|\frac{uh^R \cdot \hat n}{h^R}\right| + c^R \right) \\
  f_i          &=& \frac{A_\mathrm{face}}{2} \left( f^L_i + f^R_i + s \left( x^L_i - x^R_i \right) \right)
\end{eqnarray}
where $c$ is the local gravity wave speed and $f_i$ is a Rusanov flux.

The more sophisticated residual evaluation in RHSFunctionLocal_LS() uses a least-squares fit to a quadratic polynomial
over a neighborhood of the given element.

The mesh is read in from an ExodusII file, usually generated by Cubit.
F*/
#include <petscts.h>
#include <petscfv.h>
#include <petscdmplex.h>
#include <petscsf.h>
#include <petscblaslapack.h>

#define DIM 2                   /* Geometric dimension */
#define ALEN(a) (sizeof(a)/sizeof((a)[0]))

static PetscFunctionList PhysicsList;

/* Represents continuum physical equations. */
typedef struct _n_Physics *Physics;

/* Physical model includes boundary conditions, initial conditions, and functionals of interest. It is
 * discretization-independent, but its members depend on the scenario being solved. */
typedef struct _n_Model *Model;

/* 'User' implements a discretization of a continuous model. */
typedef struct _n_User *User;

typedef PetscErrorCode (*RiemannFunction)(const PetscReal*,const PetscReal*,const PetscScalar*,const PetscScalar*,PetscScalar*,void*);
typedef PetscErrorCode (*SolutionFunction)(Model,PetscReal,const PetscReal*,PetscScalar*,void*);
typedef PetscErrorCode (*FunctionalFunction)(Model,PetscReal,const PetscReal*,const PetscScalar*,PetscReal*,void*);
typedef PetscErrorCode (*SetupFields)(Physics,PetscSection);
static PetscErrorCode ModelSolutionSetDefault(Model,SolutionFunction,void*);
static PetscErrorCode ModelFunctionalRegister(Model,const char*,PetscInt*,FunctionalFunction,void*);
static PetscErrorCode OutputVTK(DM,const char*,PetscViewer*);

struct FieldDescription {
  const char *name;
  PetscInt dof;
};

typedef struct _n_FunctionalLink *FunctionalLink;
struct _n_FunctionalLink {
  char               *name;
  FunctionalFunction func;
  void               *ctx;
  PetscInt           offset;
  FunctionalLink     next;
};

struct _n_Physics {
  RiemannFunction riemann;
  PetscInt        dof;          /* number of degrees of freedom per cell */
  PetscReal       maxspeed;     /* kludge to pick initial time step, need to add monitoring and step control */
  void            *data;
  PetscInt        nfields;
  const struct FieldDescription *field_desc;
};

struct _n_Model {
  MPI_Comm         comm;        /* Does not do collective communicaton, but some error conditions can be collective */
  Physics          physics;
  FunctionalLink   functionalRegistry;
  PetscInt         maxComputed;
  PetscInt         numMonitored;
  FunctionalLink   *functionalMonitored;
  PetscInt         numCall;
  FunctionalLink   *functionalCall;
  SolutionFunction solution;
  void             *solutionctx;
  PetscReal        maxspeed;    /* estimate of global maximum speed (for CFL calculation) */
};

struct _n_User {
  PetscInt numSplitFaces;
  PetscInt vtkInterval;   /* For monitor */
  Model    model;
};

PETSC_STATIC_INLINE PetscScalar DotDIM(const PetscScalar *x,const PetscScalar *y)
{
  PetscInt    i;
  PetscScalar prod=0.0;

  for (i=0; i<DIM; i++) prod += x[i]*y[i];
  return prod;
}
PETSC_STATIC_INLINE PetscReal NormDIM(const PetscScalar *x) { return PetscSqrtReal(PetscAbsScalar(DotDIM(x,x))); }
PETSC_STATIC_INLINE void axDIM(const PetscScalar a,PetscScalar *x)
{
  PetscInt i;
  for (i=0; i<DIM; i++) x[i] *= a;
}
PETSC_STATIC_INLINE void waxDIM(const PetscScalar a,const PetscScalar *x, PetscScalar *w)
{
  PetscInt i;
  for (i=0; i<DIM; i++) w[i] = x[i]*a;
}
PETSC_STATIC_INLINE void NormalSplitDIM(const PetscReal *n,const PetscScalar *x,PetscScalar *xn,PetscScalar *xt)
{                               /* Split x into normal and tangential components */
  PetscInt    i;
  PetscScalar c;
  c = DotDIM(x,n)/DotDIM(n,n);
  for (i=0; i<DIM; i++) {
    xn[i] = c*n[i];
    xt[i] = x[i]-xn[i];
  }
}

PETSC_STATIC_INLINE PetscScalar Dot2(const PetscScalar *x,const PetscScalar *y) { return x[0]*y[0] + x[1]*y[1];}
PETSC_STATIC_INLINE PetscReal Norm2(const PetscScalar *x) { return PetscSqrtReal(PetscAbsScalar(Dot2(x,x)));}
PETSC_STATIC_INLINE void Normalize2(PetscScalar *x) { PetscReal a = 1./Norm2(x); x[0] *= a; x[1] *= a; }
PETSC_STATIC_INLINE void Waxpy2(PetscScalar a,const PetscScalar *x,const PetscScalar *y,PetscScalar *w) { w[0] = a*x[0] + y[0]; w[1] = a*x[1] + y[1]; }
PETSC_STATIC_INLINE void Scale2(PetscScalar a,const PetscScalar *x,PetscScalar *y) { y[0] = a*x[0]; y[1] = a*x[1]; }

PETSC_STATIC_INLINE void WaxpyD(PetscInt dim, PetscScalar a, const PetscScalar *x, const PetscScalar *y, PetscScalar *w) {PetscInt d; for (d = 0; d < dim; ++d) w[d] = a*x[d] + y[d];}
PETSC_STATIC_INLINE PetscScalar DotD(PetscInt dim, const PetscScalar *x, const PetscScalar *y) {PetscScalar sum = 0.0; PetscInt d; for (d = 0; d < dim; ++d) sum += x[d]*y[d]; return sum;}
PETSC_STATIC_INLINE PetscReal NormD(PetscInt dim, const PetscScalar *x) {return PetscSqrtReal(PetscAbsScalar(DotD(dim,x,x)));}

PETSC_STATIC_INLINE void NormalSplit(const PetscReal *n,const PetscScalar *x,PetscScalar *xn,PetscScalar *xt)
{                               /* Split x into normal and tangential components */
  Scale2(Dot2(x,n)/Dot2(n,n),n,xn);
  Waxpy2(-1,xn,x,xt);
}

/******************* Advect ********************/
typedef enum {ADVECT_SOL_TILTED,ADVECT_SOL_BUMP} AdvectSolType;
static const char *const AdvectSolTypes[] = {"TILTED","BUMP","AdvectSolType","ADVECT_SOL_",0};
typedef enum {ADVECT_SOL_BUMP_CONE,ADVECT_SOL_BUMP_COS} AdvectSolBumpType;
static const char *const AdvectSolBumpTypes[] = {"CONE","COS","AdvectSolBumpType","ADVECT_SOL_BUMP_",0};

typedef struct {
  PetscReal wind[DIM];
} Physics_Advect_Tilted;
typedef struct {
  PetscReal         center[DIM];
  PetscReal         radius;
  AdvectSolBumpType type;
} Physics_Advect_Bump;

typedef struct {
  PetscReal     inflowState;
  AdvectSolType soltype;
  union {
    Physics_Advect_Tilted tilted;
    Physics_Advect_Bump   bump;
  } sol;
  struct {
    PetscInt Error;
  } functional;
} Physics_Advect;

static const struct FieldDescription PhysicsFields_Advect[] = {{"U",1},{NULL,0}};

static PetscErrorCode PhysicsBoundary_Advect_Inflow(PetscReal time, const PetscReal *c, const PetscReal *n, const PetscScalar *xI, PetscScalar *xG, void *ctx)
{
  Physics        phys    = (Physics)ctx;
  Physics_Advect *advect = (Physics_Advect*)phys->data;

  PetscFunctionBeginUser;
  xG[0] = advect->inflowState;
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsBoundary_Advect_Outflow(PetscReal time, const PetscReal *c, const PetscReal *n, const PetscScalar *xI, PetscScalar *xG, void *ctx)
{
  PetscFunctionBeginUser;
  xG[0] = xI[0];
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsRiemann_Advect(const PetscReal *qp, const PetscReal *n, const PetscScalar *xL, const PetscScalar *xR, PetscScalar *flux, Physics phys)
{
  Physics_Advect *advect = (Physics_Advect*)phys->data;
  PetscReal      wind[DIM],wn;

  PetscFunctionBeginUser;
  switch (advect->soltype) {
  case ADVECT_SOL_TILTED: {
    Physics_Advect_Tilted *tilted = &advect->sol.tilted;
    wind[0] = tilted->wind[0];
    wind[1] = tilted->wind[1];
  } break;
  case ADVECT_SOL_BUMP:
    wind[0] = -qp[1];
    wind[1] = qp[0];
    break;
  default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"No support for solution type %s",AdvectSolBumpTypes[advect->soltype]);
  }
  wn      = Dot2(wind, n);
  flux[0] = (wn > 0 ? xL[0] : xR[0]) * wn;
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsSolution_Advect(Model mod,PetscReal time,const PetscReal *x,PetscScalar *u,void *ctx)
{
  Physics        phys    = (Physics)ctx;
  Physics_Advect *advect = (Physics_Advect*)phys->data;

  PetscFunctionBeginUser;
  switch (advect->soltype) {
  case ADVECT_SOL_TILTED: {
    PetscReal             x0[DIM];
    Physics_Advect_Tilted *tilted = &advect->sol.tilted;
    Waxpy2(-time,tilted->wind,x,x0);
    if (x0[1] > 0) u[0] = 1.*x[0] + 3.*x[1];
    else u[0] = advect->inflowState;
  } break;
  case ADVECT_SOL_BUMP: {
    Physics_Advect_Bump *bump = &advect->sol.bump;
    PetscReal           x0[DIM],v[DIM],r,cost,sint;
    cost  = PetscCosReal(time);
    sint  = PetscSinReal(time);
    x0[0] = cost*x[0] + sint*x[1];
    x0[1] = -sint*x[0] + cost*x[1];
    Waxpy2(-1,bump->center,x0,v);
    r = Norm2(v);
    switch (bump->type) {
    case ADVECT_SOL_BUMP_CONE:
      u[0] = PetscMax(1 - r/bump->radius,0);
      break;
    case ADVECT_SOL_BUMP_COS:
      u[0] = 0.5 + 0.5*PetscCosReal(PetscMin(r/bump->radius,1)*PETSC_PI);
      break;
    }
  } break;
  default: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Unknown solution type");
  }
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsFunctional_Advect(Model mod,PetscReal time,const PetscScalar *x,const PetscScalar *y,PetscReal *f,void *ctx)
{
  Physics        phys    = (Physics)ctx;
  Physics_Advect *advect = (Physics_Advect*)phys->data;
  PetscScalar    yexact[1];
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = PhysicsSolution_Advect(mod,time,x,yexact,phys);CHKERRQ(ierr);
  f[advect->functional.Error] = PetscAbsScalar(y[0]-yexact[0]);
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsCreate_Advect(PetscDS prob, Model mod,Physics phys,PetscOptions *PetscOptionsObject)
{
  Physics_Advect *advect;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  phys->field_desc = PhysicsFields_Advect;
  phys->riemann = (RiemannFunction) PhysicsRiemann_Advect;
  ierr = PetscNew(&advect);CHKERRQ(ierr);
  phys->data = advect;
  ierr = PetscOptionsHead(PetscOptionsObject,"Advect options");CHKERRQ(ierr);
  {
    PetscInt two = 2,dof = 1;
    advect->soltype = ADVECT_SOL_TILTED;
    ierr = PetscOptionsEnum("-advect_sol_type","solution type","",AdvectSolTypes,(PetscEnum)advect->soltype,(PetscEnum*)&advect->soltype,NULL);CHKERRQ(ierr);
    switch (advect->soltype) {
    case ADVECT_SOL_TILTED: {
      Physics_Advect_Tilted *tilted = &advect->sol.tilted;
      two = 2;
      tilted->wind[0] = 0.0;
      tilted->wind[1] = 1.0;
      ierr = PetscOptionsRealArray("-advect_tilted_wind","background wind vx,vy","",tilted->wind,&two,NULL);CHKERRQ(ierr);
      advect->inflowState = -2.0;
      ierr = PetscOptionsRealArray("-advect_tilted_inflow","Inflow state","",&advect->inflowState,&dof,NULL);CHKERRQ(ierr);
      phys->maxspeed = Norm2(tilted->wind);
    } break;
    case ADVECT_SOL_BUMP: {
      Physics_Advect_Bump *bump = &advect->sol.bump;
      two = 2;
      bump->center[0] = 2.;
      bump->center[1] = 0.;
      ierr = PetscOptionsRealArray("-advect_bump_center","location of center of bump x,y","",bump->center,&two,NULL);CHKERRQ(ierr);
      bump->radius = 0.9;
      ierr = PetscOptionsReal("-advect_bump_radius","radius of bump","",bump->radius,&bump->radius,NULL);CHKERRQ(ierr);
      bump->type = ADVECT_SOL_BUMP_CONE;
      ierr = PetscOptionsEnum("-advect_bump_type","type of bump","",AdvectSolBumpTypes,(PetscEnum)bump->type,(PetscEnum*)&bump->type,NULL);CHKERRQ(ierr);
      phys->maxspeed = 3.;       /* radius of mesh, kludge */
    } break;
    }
  }
  ierr = PetscOptionsTail();CHKERRQ(ierr);
  {
    const PetscInt inflowids[] = {100,200,300},outflowids[] = {101};
    DMLabel        label;

    ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
    /* Register "canned" boundary conditions and defaults for where to apply. */
    ierr = PetscDSAddBoundary(prob, PETSC_TRUE, "inflow",  label, ALEN(inflowids),  inflowids,  0, 0, NULL, (void (*)()) PhysicsBoundary_Advect_Inflow,  NULL,  phys, NULL);CHKERRQ(ierr);
    ierr = PetscDSAddBoundary(prob, PETSC_TRUE, "outflow", label, ALEN(outflowids), outflowids, 0, 0, NULL, (void (*)()) PhysicsBoundary_Advect_Outflow, NULL, phys, NULL);CHKERRQ(ierr);
    /* Initial/transient solution with default boundary conditions */
    ierr = ModelSolutionSetDefault(mod,PhysicsSolution_Advect,phys);CHKERRQ(ierr);
    /* Register "canned" functionals */
    ierr = ModelFunctionalRegister(mod,"Error",&advect->functional.Error,PhysicsFunctional_Advect,phys);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

/******************* Shallow Water ********************/
typedef struct {
  PetscReal gravity;
  PetscReal boundaryHeight;
  struct {
    PetscInt Height;
    PetscInt Speed;
    PetscInt Energy;
  } functional;
} Physics_SW;
typedef struct {
  PetscScalar vals[0];
  PetscScalar h;
  PetscScalar uh[DIM];
} SWNode;

static const struct FieldDescription PhysicsFields_SW[] = {{"Height",1},{"Momentum",DIM},{NULL,0}};

/*
 * h_t + div(uh) = 0
 * (uh)_t + div (u\otimes uh + g h^2 / 2 I) = 0
 *
 * */
static PetscErrorCode SWFlux(Physics phys,const PetscReal *n,const SWNode *x,SWNode *f)
{
  Physics_SW  *sw = (Physics_SW*)phys->data;
  PetscScalar uhn,u[DIM];
  PetscInt    i;

  PetscFunctionBeginUser;
  Scale2(1./x->h,x->uh,u);
  uhn  = Dot2(x->uh,n);
  f->h = uhn;
  for (i=0; i<DIM; i++) f->uh[i] = u[i] * uhn + sw->gravity * PetscSqr(x->h) * n[i];
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsBoundary_SW_Wall(PetscReal time, const PetscReal *c, const PetscReal *n, const PetscScalar *xI, PetscScalar *xG, void *ctx)
{
  PetscFunctionBeginUser;
  xG[0] = xI[0];
  xG[1] = -xI[1];
  xG[2] = -xI[2];
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsRiemann_SW(const PetscReal *qp, const PetscReal *n, const PetscScalar *xL, const PetscScalar *xR, PetscScalar *flux, Physics phys)
{
  Physics_SW   *sw = (Physics_SW*)phys->data;
  PetscReal    cL,cR,speed,nn[DIM];
  const SWNode *uL = (const SWNode*)xL,*uR = (const SWNode*)xR;
  SWNode       fL,fR;
  PetscInt     i;

  PetscFunctionBeginUser;
  if (uL->h < 0 || uR->h < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Reconstructed thickness is negative");
  nn[0] = n[0];
  nn[1] = n[1];
  Normalize2(nn);
  SWFlux(phys,nn,uL,&fL);
  SWFlux(phys,nn,uR,&fR);
  cL    = PetscSqrtReal(sw->gravity*PetscRealPart(uL->h));
  cR    = PetscSqrtReal(sw->gravity*PetscRealPart(uR->h)); /* gravity wave speed */
  speed = PetscMax(PetscAbsScalar(Dot2(uL->uh,nn)/uL->h) + cL,PetscAbsScalar(Dot2(uR->uh,nn)/uR->h) + cR);
  for (i=0; i<1+DIM; i++) flux[i] = (0.5*(fL.vals[i] + fR.vals[i]) + 0.5*speed*(xL[i] - xR[i])) * Norm2(n);
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsSolution_SW(Model mod,PetscReal time,const PetscReal *x,PetscScalar *u,void *ctx)
{
  PetscReal dx[2],r,sigma;

  PetscFunctionBeginUser;
  if (time != 0.0) SETERRQ1(mod->comm,PETSC_ERR_SUP,"No solution known for time %g",(double)time);
  dx[0] = x[0] - 1.5;
  dx[1] = x[1] - 1.0;
  r     = Norm2(dx);
  sigma = 0.5;
  u[0]  = 1 + 2*PetscExpScalar(-PetscSqr(r)/(2*PetscSqr(sigma)));
  u[1]  = 0.0;
  u[2]  = 0.0;
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsFunctional_SW(Model mod,PetscReal time,const PetscReal *coord,const PetscScalar *xx,PetscReal *f,void *ctx)
{
  Physics      phys = (Physics)ctx;
  Physics_SW   *sw  = (Physics_SW*)phys->data;
  const SWNode *x   = (const SWNode*)xx;
  PetscScalar  u[2];
  PetscReal    h;

  PetscFunctionBeginUser;
  h = PetscRealPart(x->h);
  Scale2(1./x->h,x->uh,u);
  f[sw->functional.Height] = h;
  f[sw->functional.Speed]  = Norm2(u) + PetscSqrtReal(sw->gravity*h);
  f[sw->functional.Energy] = 0.5*(Dot2(x->uh,u) + sw->gravity*PetscSqr(h));
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsCreate_SW(PetscDS prob, Model mod,Physics phys,PetscOptions *PetscOptionsObject)
{
  Physics_SW     *sw;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  phys->field_desc = PhysicsFields_SW;
  phys->riemann = (RiemannFunction) PhysicsRiemann_SW;
  ierr          = PetscNew(&sw);CHKERRQ(ierr);
  phys->data    = sw;
  ierr          = PetscOptionsHead(PetscOptionsObject,"SW options");CHKERRQ(ierr);
  {
    sw->gravity = 1.0;
    ierr = PetscOptionsReal("-sw_gravity","Gravitational constant","",sw->gravity,&sw->gravity,NULL);CHKERRQ(ierr);
  }
  ierr = PetscOptionsTail();CHKERRQ(ierr);
  phys->maxspeed = PetscSqrtReal(2.0*sw->gravity); /* Mach 1 for depth of 2 */

  {
    const PetscInt wallids[] = {100,101,200,300};
    DMLabel        label;

    ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
    ierr = PetscDSAddBoundary(prob, PETSC_TRUE, "wall", label, ALEN(wallids), wallids, 0, 0, NULL, (void (*)()) PhysicsBoundary_SW_Wall, NULL, phys, NULL);CHKERRQ(ierr);
    ierr = ModelSolutionSetDefault(mod,PhysicsSolution_SW,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Height",&sw->functional.Height,PhysicsFunctional_SW,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Speed",&sw->functional.Speed,PhysicsFunctional_SW,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Energy",&sw->functional.Energy,PhysicsFunctional_SW,phys);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

/******************* Euler ********************/
typedef struct {
  PetscScalar vals[0];
  PetscScalar r;
  PetscScalar ru[DIM];
  PetscScalar e;
} EulerNode;
typedef PetscErrorCode (*EquationOfState)(const PetscReal*, const EulerNode*, PetscScalar*);
typedef struct {
  PetscInt        npars;
  PetscReal       pars[DIM];
  EquationOfState pressure;
  EquationOfState sound;
  struct {
    PetscInt Density;
    PetscInt Momentum;
    PetscInt Energy;
    PetscInt Pressure;
    PetscInt Speed;
  } monitor;
} Physics_Euler;

static const struct FieldDescription PhysicsFields_Euler[] = {{"Density",1},{"Momentum",DIM},{"Energy",1},{NULL,0}};

static PetscErrorCode Pressure_PG(const PetscReal *pars,const EulerNode *x,PetscScalar *p)
{
  PetscScalar ru2;

  PetscFunctionBeginUser;
  ru2  = DotDIM(x->ru,x->ru);
  ru2 /= x->r;
  /* kinematic dof = params[0] */
  (*p)=2.0*(x->e-0.5*ru2)/pars[0];
  PetscFunctionReturn(0);
}

static PetscErrorCode SpeedOfSound_PG(const PetscReal *pars,const EulerNode *x,PetscScalar *c)
{
  PetscScalar p;

  PetscFunctionBeginUser;
  /* TODO remove direct usage of Pressure_PG */
  Pressure_PG(pars,x,&p);
  /* TODO check the sign of p */
  /* pars[1] = heat capacity ratio */
  (*c)=PetscSqrtScalar(pars[1]*p/x->r);
  PetscFunctionReturn(0);
}

/*
 * x = (rho,rho*(u_1),...,rho*e)^T
 * x_t+div(f_1(x))+...+div(f_DIM(x)) = 0
 *
 * f_i(x) = u_i*x+(0,0,...,p,...,p*u_i)^T
 *
 * */
static PetscErrorCode EulerFlux(Physics phys,const PetscReal *n,const EulerNode *x,EulerNode *f)
{
  Physics_Euler *eu = (Physics_Euler*)phys->data;
  PetscScalar   u,nu,p;
  PetscInt      i;

  PetscFunctionBeginUser;
  u  = DotDIM(x->ru,x->ru);
  u /= (x->r * x->r);
  nu = DotDIM(x->ru,n);
  /* TODO check the sign of p */
  eu->pressure(eu->pars,x,&p);
  f->r = nu * x->r;
  for (i=0; i<DIM; i++) f->ru[i] = nu * x->ru[i] + n[i]*p;
  f->e = nu*(x->e+p);
  PetscFunctionReturn(0);
}

/* PetscReal* => EulerNode* conversion */
static PetscErrorCode PhysicsBoundary_Euler_Wall(PetscReal time, const PetscReal *c, const PetscReal *n, const PetscScalar *xI, PetscScalar *xG, void *ctx)
{
  PetscInt    i;
  PetscScalar xn[DIM],xt[DIM];

  PetscFunctionBeginUser;
  xG[0] = xI[0];
  NormalSplitDIM(n,xI+1,xn,xt);
  for (i=0; i<DIM; i++) xG[i+1] = -xn[i]+xt[i];
  xG[DIM+1] = xI[DIM+1];
  PetscFunctionReturn(0);
}

/* PetscReal* => EulerNode* conversion */
static PetscErrorCode PhysicsRiemann_Euler_Rusanov(const PetscReal *qp, const PetscReal *n, const PetscScalar *xL, const PetscScalar *xR, PetscScalar *flux, Physics phys)
{
  Physics_Euler   *eu = (Physics_Euler*)phys->data;
  PetscScalar     cL,cR,speed;
  const EulerNode *uL = (const EulerNode*)xL,*uR = (const EulerNode*)xR;
  EulerNode       fL,fR;
  PetscInt        i;

  PetscFunctionBeginUser;
  if (uL->r < 0 || uR->r < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Reconstructed density is negative");
  EulerFlux(phys,n,uL,&fL);
  EulerFlux(phys,n,uR,&fR);
  eu->sound(eu->pars,uL,&cL);
  eu->sound(eu->pars,uR,&cR);
  speed = PetscMax(cL,cR)+PetscMax(PetscAbsScalar(DotDIM(uL->ru,n)/NormDIM(n)),PetscAbsScalar(DotDIM(uR->ru,n)/NormDIM(n)));
  for (i=0; i<2+DIM; i++) flux[i] = 0.5*(fL.vals[i]+fR.vals[i])+0.5*speed*(xL[i]-xR[i]);
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsSolution_Euler(Model mod,PetscReal time,const PetscReal *x,PetscScalar *u,void *ctx)
{
  PetscInt i;

  PetscFunctionBeginUser;
  if (time != 0.0) SETERRQ1(mod->comm,PETSC_ERR_SUP,"No solution known for time %g",(double)time);
  u[0]     = 1.0;
  u[DIM+1] = 1.0+PetscAbsReal(x[0]);
  for (i=1; i<DIM+1; i++) u[i] = 0.0;
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsFunctional_Euler(Model mod,PetscReal time,const PetscReal *coord,const PetscScalar *xx,PetscReal *f,void *ctx)
{
  Physics         phys = (Physics)ctx;
  Physics_Euler   *eu  = (Physics_Euler*)phys->data;
  const EulerNode *x   = (const EulerNode*)xx;
  PetscScalar     p;

  PetscFunctionBeginUser;
  f[eu->monitor.Density]  = x->r;
  f[eu->monitor.Momentum] = NormDIM(x->ru);
  f[eu->monitor.Energy]   = x->e;
  f[eu->monitor.Speed]    = NormDIM(x->ru)/x->r;
  eu->pressure(eu->pars, x, &p);
  f[eu->monitor.Pressure] = p;
  PetscFunctionReturn(0);
}

static PetscErrorCode PhysicsCreate_Euler(PetscDS prob, Model mod,Physics phys,PetscOptions *PetscOptionsObject)
{
  Physics_Euler   *eu;
  PetscErrorCode  ierr;

  PetscFunctionBeginUser;
  phys->field_desc = PhysicsFields_Euler;
  phys->riemann = (RiemannFunction) PhysicsRiemann_Euler_Rusanov;
  ierr = PetscNew(&eu);CHKERRQ(ierr);
  phys->data    = eu;
  ierr = PetscOptionsHead(PetscOptionsObject,"Euler options");CHKERRQ(ierr);
  {
    eu->pars[0] = 3.0;
    eu->pars[1] = 1.67;
    ierr = PetscOptionsReal("-eu_f","Degrees of freedom","",eu->pars[0],&eu->pars[0],NULL);CHKERRQ(ierr);
    ierr = PetscOptionsReal("-eu_gamma","Heat capacity ratio","",eu->pars[1],&eu->pars[1],NULL);CHKERRQ(ierr);
  }
  ierr = PetscOptionsTail();CHKERRQ(ierr);
  eu->pressure = Pressure_PG;
  eu->sound    = SpeedOfSound_PG;
  phys->maxspeed = 1.0;
  {
    const PetscInt wallids[] = {100,101,200,300};
    DMLabel        label;

    ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
    ierr = PetscDSAddBoundary(prob, PETSC_TRUE, "wall", label, ALEN(wallids), wallids, 0, 0, NULL, (void (*)()) PhysicsBoundary_Euler_Wall, NULL, phys, NULL);CHKERRQ(ierr);
    ierr = ModelSolutionSetDefault(mod,PhysicsSolution_Euler,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Speed",&eu->monitor.Speed,PhysicsFunctional_Euler,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Energy",&eu->monitor.Energy,PhysicsFunctional_Euler,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Density",&eu->monitor.Density,PhysicsFunctional_Euler,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Momentum",&eu->monitor.Momentum,PhysicsFunctional_Euler,phys);CHKERRQ(ierr);
    ierr = ModelFunctionalRegister(mod,"Pressure",&eu->monitor.Pressure,PhysicsFunctional_Euler,phys);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

PetscErrorCode ConstructCellBoundary(DM dm, User user)
{
  const char     *name   = "Cell Sets";
  const char     *bdname = "split faces";
  IS             regionIS, innerIS;
  const PetscInt *regions, *cells;
  PetscInt       numRegions, innerRegion, numCells, c;
  PetscInt       cStart, cEnd, cEndInterior, fStart, fEnd;
  PetscBool      hasLabel;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
  ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
  ierr = DMPlexGetGhostCellStratum(dm, &cEndInterior, NULL);CHKERRQ(ierr);

  ierr = DMHasLabel(dm, name, &hasLabel);CHKERRQ(ierr);
  if (!hasLabel) PetscFunctionReturn(0);
  ierr = DMGetLabelSize(dm, name, &numRegions);CHKERRQ(ierr);
  if (numRegions != 2) PetscFunctionReturn(0);
  /* Get the inner id */
  ierr = DMGetLabelIdIS(dm, name, &regionIS);CHKERRQ(ierr);
  ierr = ISGetIndices(regionIS, &regions);CHKERRQ(ierr);
  innerRegion = regions[0];
  ierr = ISRestoreIndices(regionIS, &regions);CHKERRQ(ierr);
  ierr = ISDestroy(&regionIS);CHKERRQ(ierr);
  /* Find the faces between cells in different regions, could call DMPlexCreateNeighborCSR() */
  ierr = DMGetStratumIS(dm, name, innerRegion, &innerIS);CHKERRQ(ierr);
  ierr = ISGetLocalSize(innerIS, &numCells);CHKERRQ(ierr);
  ierr = ISGetIndices(innerIS, &cells);CHKERRQ(ierr);
  ierr = DMCreateLabel(dm, bdname);CHKERRQ(ierr);
  for (c = 0; c < numCells; ++c) {
    const PetscInt cell = cells[c];
    const PetscInt *faces;
    PetscInt       numFaces, f;

    if ((cell < cStart) || (cell >= cEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_LIB, "Got invalid point %d which is not a cell", cell);
    ierr = DMPlexGetConeSize(dm, cell, &numFaces);CHKERRQ(ierr);
    ierr = DMPlexGetCone(dm, cell, &faces);CHKERRQ(ierr);
    for (f = 0; f < numFaces; ++f) {
      const PetscInt face = faces[f];
      const PetscInt *neighbors;
      PetscInt       nC, regionA, regionB;

      if ((face < fStart) || (face >= fEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_LIB, "Got invalid point %d which is not a face", face);
      ierr = DMPlexGetSupportSize(dm, face, &nC);CHKERRQ(ierr);
      if (nC != 2) continue;
      ierr = DMPlexGetSupport(dm, face, &neighbors);CHKERRQ(ierr);
      if ((neighbors[0] >= cEndInterior) || (neighbors[1] >= cEndInterior)) continue;
      if ((neighbors[0] < cStart) || (neighbors[0] >= cEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_LIB, "Got invalid point %d which is not a cell", neighbors[0]);
      if ((neighbors[1] < cStart) || (neighbors[1] >= cEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_LIB, "Got invalid point %d which is not a cell", neighbors[1]);
      ierr = DMGetLabelValue(dm, name, neighbors[0], &regionA);CHKERRQ(ierr);
      ierr = DMGetLabelValue(dm, name, neighbors[1], &regionB);CHKERRQ(ierr);
      if (regionA < 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid label %s: Cell %d has no value", name, neighbors[0]);
      if (regionB < 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid label %s: Cell %d has no value", name, neighbors[1]);
      if (regionA != regionB) {
        ierr = DMSetLabelValue(dm, bdname, faces[f], 1);CHKERRQ(ierr);
      }
    }
  }
  ierr = ISRestoreIndices(innerIS, &cells);CHKERRQ(ierr);
  ierr = ISDestroy(&innerIS);CHKERRQ(ierr);
  {
    DMLabel label;

    ierr = DMGetLabel(dm, bdname, &label);CHKERRQ(ierr);
    ierr = DMLabelView(label, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

/* Right now, I have just added duplicate faces, which see both cells. We can
- Add duplicate vertices and decouple the face cones
- Disconnect faces from cells across the rotation gap
*/
PetscErrorCode SplitFaces(DM *dmSplit, const char labelName[], User user)
{
  DM             dm = *dmSplit, sdm;
  PetscSF        sfPoint, gsfPoint;
  PetscSection   coordSection, newCoordSection;
  Vec            coordinates;
  IS             idIS;
  const PetscInt *ids;
  PetscInt       *newpoints;
  PetscInt       dim, depth, maxConeSize, maxSupportSize, numLabels, numGhostCells;
  PetscInt       numFS, fs, pStart, pEnd, p, cEnd, cEndInterior, vStart, vEnd, v, fStart, fEnd, newf, d, l;
  PetscBool      hasLabel;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = DMHasLabel(dm, labelName, &hasLabel);CHKERRQ(ierr);
  if (!hasLabel) PetscFunctionReturn(0);
  ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
  ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
  ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
  ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);

  ierr = DMGetLabelIdIS(dm, labelName, &idIS);CHKERRQ(ierr);
  ierr = ISGetLocalSize(idIS, &numFS);CHKERRQ(ierr);
  ierr = ISGetIndices(idIS, &ids);CHKERRQ(ierr);

  user->numSplitFaces = 0;
  for (fs = 0; fs < numFS; ++fs) {
    PetscInt numBdFaces;

    ierr = DMGetStratumSize(dm, labelName, ids[fs], &numBdFaces);CHKERRQ(ierr);
    user->numSplitFaces += numBdFaces;
  }
  ierr  = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
  pEnd += user->numSplitFaces;
  ierr  = DMPlexSetChart(sdm, pStart, pEnd);CHKERRQ(ierr);
  ierr  = DMPlexGetGhostCellStratum(dm, &cEndInterior, NULL);CHKERRQ(ierr);
  ierr  = DMPlexSetGhostCellStratum(sdm, cEndInterior, PETSC_DETERMINE);CHKERRQ(ierr);
  ierr  = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
  numGhostCells = cEnd - cEndInterior;
  /* Set cone and support sizes */
  ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
  for (d = 0; d <= depth; ++d) {
    ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
    for (p = pStart; p < pEnd; ++p) {
      PetscInt newp = p;
      PetscInt size;

      ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
      ierr = DMPlexSetConeSize(sdm, newp, size);CHKERRQ(ierr);
      ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
      ierr = DMPlexSetSupportSize(sdm, newp, size);CHKERRQ(ierr);
    }
  }
  ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
  for (fs = 0, newf = fEnd; fs < numFS; ++fs) {
    IS             faceIS;
    const PetscInt *faces;
    PetscInt       numFaces, f;

    ierr = DMGetStratumIS(dm, labelName, ids[fs], &faceIS);CHKERRQ(ierr);
    ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
    ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
    for (f = 0; f < numFaces; ++f, ++newf) {
      PetscInt size;

      /* Right now I think that both faces should see both cells */
      ierr = DMPlexGetConeSize(dm, faces[f], &size);CHKERRQ(ierr);
      ierr = DMPlexSetConeSize(sdm, newf, size);CHKERRQ(ierr);
      ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
      ierr = DMPlexSetSupportSize(sdm, newf, size);CHKERRQ(ierr);
    }
    ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
    ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
  }
  ierr = DMSetUp(sdm);CHKERRQ(ierr);
  /* Set cones and supports */
  ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
  ierr = PetscMalloc1(PetscMax(maxConeSize, maxSupportSize), &newpoints);CHKERRQ(ierr);
  ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
  for (p = pStart; p < pEnd; ++p) {
    const PetscInt *points, *orientations;
    PetscInt       size, i, newp = p;

    ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
    ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
    ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
    for (i = 0; i < size; ++i) newpoints[i] = points[i];
    ierr = DMPlexSetCone(sdm, newp, newpoints);CHKERRQ(ierr);
    ierr = DMPlexSetConeOrientation(sdm, newp, orientations);CHKERRQ(ierr);
    ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
    ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
    for (i = 0; i < size; ++i) newpoints[i] = points[i];
    ierr = DMPlexSetSupport(sdm, newp, newpoints);CHKERRQ(ierr);
  }
  ierr = PetscFree(newpoints);CHKERRQ(ierr);
  for (fs = 0, newf = fEnd; fs < numFS; ++fs) {
    IS             faceIS;
    const PetscInt *faces;
    PetscInt       numFaces, f;

    ierr = DMGetStratumIS(dm, labelName, ids[fs], &faceIS);CHKERRQ(ierr);
    ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
    ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
    for (f = 0; f < numFaces; ++f, ++newf) {
      const PetscInt *points;

      ierr = DMPlexGetCone(dm, faces[f], &points);CHKERRQ(ierr);
      ierr = DMPlexSetCone(sdm, newf, points);CHKERRQ(ierr);
      ierr = DMPlexGetSupport(dm, faces[f], &points);CHKERRQ(ierr);
      ierr = DMPlexSetSupport(sdm, newf, points);CHKERRQ(ierr);
    }
    ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
    ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
  }
  ierr = ISRestoreIndices(idIS, &ids);CHKERRQ(ierr);
  ierr = ISDestroy(&idIS);CHKERRQ(ierr);
  ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
  /* Convert coordinates */
  ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
  ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
  ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
  ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
  ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
  ierr = PetscSectionSetChart(newCoordSection, vStart, vEnd);CHKERRQ(ierr);
  for (v = vStart; v < vEnd; ++v) {
    ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
    ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
  }
  ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
  ierr = DMSetCoordinateSection(sdm, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
  ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr); /* relinquish our reference */
  ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
  ierr = DMSetCoordinatesLocal(sdm, coordinates);CHKERRQ(ierr);
  /* Convert labels */
  ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
  for (l = 0; l < numLabels; ++l) {
    const char *lname;
    PetscBool  isDepth;

    ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
    ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
    if (isDepth) continue;
    ierr = DMCreateLabel(sdm, lname);CHKERRQ(ierr);
    ierr = DMGetLabelIdIS(dm, lname, &idIS);CHKERRQ(ierr);
    ierr = ISGetLocalSize(idIS, &numFS);CHKERRQ(ierr);
    ierr = ISGetIndices(idIS, &ids);CHKERRQ(ierr);
    for (fs = 0; fs < numFS; ++fs) {
      IS             pointIS;
      const PetscInt *points;
      PetscInt       numPoints;

      ierr = DMGetStratumIS(dm, lname, ids[fs], &pointIS);CHKERRQ(ierr);
      ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
      ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
      for (p = 0; p < numPoints; ++p) {
        PetscInt newpoint = points[p];

        ierr = DMSetLabelValue(sdm, lname, newpoint, ids[fs]);CHKERRQ(ierr);
      }
      ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
      ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
    }
    ierr = ISRestoreIndices(idIS, &ids);CHKERRQ(ierr);
    ierr = ISDestroy(&idIS);CHKERRQ(ierr);
  }
  /* Convert pointSF */
  const PetscSFNode *remotePoints;
  PetscSFNode       *gremotePoints;
  const PetscInt    *localPoints;
  PetscInt          *glocalPoints,*newLocation,*newRemoteLocation;
  PetscInt          numRoots, numLeaves;
  PetscMPIInt       size;

  ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
  ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
  ierr = DMGetPointSF(sdm, &gsfPoint);CHKERRQ(ierr);
  ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
  ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
  if (numRoots >= 0) {
    ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
    for (l=0; l<numRoots; l++) newLocation[l] = l; /* + (l >= cEnd ? numGhostCells : 0); */
    ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation,MPI_REPLACE);CHKERRQ(ierr);
    ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation,MPI_REPLACE);CHKERRQ(ierr);
    ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
    ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
    for (l = 0; l < numLeaves; ++l) {
      glocalPoints[l]        = localPoints[l]; /* localPoints[l] >= cEnd ? localPoints[l] + numGhostCells : localPoints[l]; */
      gremotePoints[l].rank  = remotePoints[l].rank;
      gremotePoints[l].index = newRemoteLocation[localPoints[l]];
    }
    ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
    ierr = PetscSFSetGraph(gsfPoint, numRoots+numGhostCells, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
  }
  ierr     = DMDestroy(dmSplit);CHKERRQ(ierr);
  *dmSplit = sdm;
  PetscFunctionReturn(0);
}

PetscErrorCode CreatePartitionVec(DM dm, DM *dmCell, Vec *partition)
{
  PetscSF        sfPoint;
  PetscSection   coordSection;
  Vec            coordinates;
  PetscSection   sectionCell;
  PetscScalar    *part;
  PetscInt       cStart, cEnd, c;
  PetscMPIInt    rank;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
  ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
  ierr = DMClone(dm, dmCell);CHKERRQ(ierr);
  ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
  ierr = DMSetPointSF(*dmCell, sfPoint);CHKERRQ(ierr);
  ierr = DMSetCoordinateSection(*dmCell, PETSC_DETERMINE, coordSection);CHKERRQ(ierr);
  ierr = DMSetCoordinatesLocal(*dmCell, coordinates);CHKERRQ(ierr);
  ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
  ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &sectionCell);CHKERRQ(ierr);
  ierr = DMPlexGetHeightStratum(*dmCell, 0, &cStart, &cEnd);CHKERRQ(ierr);
  ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
  for (c = cStart; c < cEnd; ++c) {
    ierr = PetscSectionSetDof(sectionCell, c, 1);CHKERRQ(ierr);
  }
  ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
  ierr = DMSetLocalSection(*dmCell, sectionCell);CHKERRQ(ierr);
  ierr = PetscSectionDestroy(&sectionCell);CHKERRQ(ierr);
  ierr = DMCreateLocalVector(*dmCell, partition);CHKERRQ(ierr);
  ierr = PetscObjectSetName((PetscObject)*partition, "partition");CHKERRQ(ierr);
  ierr = VecGetArray(*partition, &part);CHKERRQ(ierr);
  for (c = cStart; c < cEnd; ++c) {
    PetscScalar *p;

    ierr = DMPlexPointLocalRef(*dmCell, c, part, &p);CHKERRQ(ierr);
    p[0] = rank;
  }
  ierr = VecRestoreArray(*partition, &part);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

PetscErrorCode CreateMassMatrix(DM dm, Vec *massMatrix, User user)
{
  DM                plex, dmMass, dmFace, dmCell, dmCoord;
  PetscSection      coordSection;
  Vec               coordinates, facegeom, cellgeom;
  PetscSection      sectionMass;
  PetscScalar       *m;
  const PetscScalar *fgeom, *cgeom, *coords;
  PetscInt          vStart, vEnd, v;
  PetscErrorCode    ierr;

  PetscFunctionBeginUser;
  ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
  ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
  ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
  ierr = DMClone(dm, &dmMass);CHKERRQ(ierr);
  ierr = DMSetCoordinateSection(dmMass, PETSC_DETERMINE, coordSection);CHKERRQ(ierr);
  ierr = DMSetCoordinatesLocal(dmMass, coordinates);CHKERRQ(ierr);
  ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &sectionMass);CHKERRQ(ierr);
  ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
  ierr = PetscSectionSetChart(sectionMass, vStart, vEnd);CHKERRQ(ierr);
  for (v = vStart; v < vEnd; ++v) {
    PetscInt numFaces;

    ierr = DMPlexGetSupportSize(dmMass, v, &numFaces);CHKERRQ(ierr);
    ierr = PetscSectionSetDof(sectionMass, v, numFaces*numFaces);CHKERRQ(ierr);
  }
  ierr = PetscSectionSetUp(sectionMass);CHKERRQ(ierr);
  ierr = DMSetLocalSection(dmMass, sectionMass);CHKERRQ(ierr);
  ierr = PetscSectionDestroy(&sectionMass);CHKERRQ(ierr);
  ierr = DMGetLocalVector(dmMass, massMatrix);CHKERRQ(ierr);
  ierr = VecGetArray(*massMatrix, &m);CHKERRQ(ierr);
  ierr = DMPlexGetGeometryFVM(plex, &facegeom, &cellgeom, NULL);CHKERRQ(ierr);
  ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
  ierr = VecGetArrayRead(facegeom, &fgeom);CHKERRQ(ierr);
  ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
  ierr = VecGetArrayRead(cellgeom, &cgeom);CHKERRQ(ierr);
  ierr = DMGetCoordinateDM(dm, &dmCoord);CHKERRQ(ierr);
  ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
  for (v = vStart; v < vEnd; ++v) {
    const PetscInt    *faces;
    const PetscFVFaceGeom    *fgA, *fgB, *cg;
    const PetscScalar *vertex;
    PetscInt          numFaces, sides[2], f, g;

    ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex);CHKERRQ(ierr);
    ierr = DMPlexGetSupportSize(dmMass, v, &numFaces);CHKERRQ(ierr);
    ierr = DMPlexGetSupport(dmMass, v, &faces);CHKERRQ(ierr);
    for (f = 0; f < numFaces; ++f) {
      sides[0] = faces[f];
      ierr = DMPlexPointLocalRead(dmFace, faces[f], fgeom, &fgA);CHKERRQ(ierr);
      for (g = 0; g < numFaces; ++g) {
        const PetscInt *cells = NULL;
        PetscReal      area   = 0.0;
        PetscInt       numCells;

        sides[1] = faces[g];
        ierr = DMPlexPointLocalRead(dmFace, faces[g], fgeom, &fgB);CHKERRQ(ierr);
        ierr = DMPlexGetJoin(dmMass, 2, sides, &numCells, &cells);CHKERRQ(ierr);
        if (numCells != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Invalid join for faces");
        ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cg);CHKERRQ(ierr);
        area += PetscAbsScalar((vertex[0] - cg->centroid[0])*(fgA->centroid[1] - cg->centroid[1]) - (vertex[1] - cg->centroid[1])*(fgA->centroid[0] - cg->centroid[0]));
        area += PetscAbsScalar((vertex[0] - cg->centroid[0])*(fgB->centroid[1] - cg->centroid[1]) - (vertex[1] - cg->centroid[1])*(fgB->centroid[0] - cg->centroid[0]));
        m[f*numFaces+g] = Dot2(fgA->normal, fgB->normal)*area*0.5;
        ierr = DMPlexRestoreJoin(dmMass, 2, sides, &numCells, &cells);CHKERRQ(ierr);
      }
    }
  }
  ierr = VecRestoreArrayRead(facegeom, &fgeom);CHKERRQ(ierr);
  ierr = VecRestoreArrayRead(cellgeom, &cgeom);CHKERRQ(ierr);
  ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
  ierr = VecRestoreArray(*massMatrix, &m);CHKERRQ(ierr);
  ierr = DMDestroy(&dmMass);CHKERRQ(ierr);
  ierr = DMDestroy(&plex);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

PetscErrorCode SetUpLocalSpace(DM dm, User user)
{
  PetscSection   stateSection;
  Physics        phys;
  PetscInt       dof = user->model->physics->dof, *cind, d, stateSize, cStart, cEnd, cEndInterior, c, i;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
  ierr = DMPlexGetGhostCellStratum(dm, &cEndInterior, NULL);CHKERRQ(ierr);
  ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &stateSection);CHKERRQ(ierr);
  phys = user->model->physics;
  ierr = PetscSectionSetNumFields(stateSection,phys->nfields);CHKERRQ(ierr);
  for (i=0; i<phys->nfields; i++) {
    ierr = PetscSectionSetFieldName(stateSection,i,phys->field_desc[i].name);CHKERRQ(ierr);
    ierr = PetscSectionSetFieldComponents(stateSection,i,phys->field_desc[i].dof);CHKERRQ(ierr);
  }
  ierr = PetscSectionSetChart(stateSection, cStart, cEnd);CHKERRQ(ierr);
  for (c = cStart; c < cEnd; ++c) {
    for (i=0; i<phys->nfields; i++) {
      ierr = PetscSectionSetFieldDof(stateSection,c,i,phys->field_desc[i].dof);CHKERRQ(ierr);
    }
    ierr = PetscSectionSetDof(stateSection, c, dof);CHKERRQ(ierr);
  }
  for (c = cEndInterior; c < cEnd; ++c) {
    ierr = PetscSectionSetConstraintDof(stateSection, c, dof);CHKERRQ(ierr);
  }
  ierr = PetscSectionSetUp(stateSection);CHKERRQ(ierr);
  ierr = PetscMalloc1(dof, &cind);CHKERRQ(ierr);
  for (d = 0; d < dof; ++d) cind[d] = d;
#if 0
  for (c = cStart; c < cEnd; ++c) {
    PetscInt val;

    ierr = DMGetLabelValue(dm, "vtk", c, &val);CHKERRQ(ierr);
    if (val < 0) {ierr = PetscSectionSetConstraintIndices(stateSection, c, cind);CHKERRQ(ierr);}
  }
#endif
  for (c = cEndInterior; c < cEnd; ++c) {
    ierr = PetscSectionSetConstraintIndices(stateSection, c, cind);CHKERRQ(ierr);
  }
  ierr = PetscFree(cind);CHKERRQ(ierr);
  ierr = PetscSectionGetStorageSize(stateSection, &stateSize);CHKERRQ(ierr);
  ierr = DMSetLocalSection(dm,stateSection);CHKERRQ(ierr);
  ierr = PetscSectionDestroy(&stateSection);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

#if 0
PetscErrorCode SetUpBoundaries(DM dm, User user)
{
  Model          mod = user->model;
  PetscErrorCode ierr;
  BoundaryLink   b;

  PetscFunctionBeginUser;
  ierr = PetscOptionsBegin(PetscObjectComm((PetscObject)dm),NULL,"Boundary condition options","");CHKERRQ(ierr);
  for (b = mod->boundary; b; b=b->next) {
    char      optname[512];
    PetscInt  ids[512],len = 512;
    PetscBool flg;
    ierr = PetscSNPrintf(optname,sizeof optname,"-bc_%s",b->name);CHKERRQ(ierr);
    ierr = PetscMemzero(ids,sizeof(ids));CHKERRQ(ierr);
    ierr = PetscOptionsIntArray(optname,"List of boundary IDs","",ids,&len,&flg);CHKERRQ(ierr);
    if (flg) {
      /* TODO: check all IDs to make sure they exist in the mesh */
      ierr      = PetscFree(b->ids);CHKERRQ(ierr);
      b->numids = len;
      ierr      = PetscMalloc1(len,&b->ids);CHKERRQ(ierr);
      ierr      = PetscArraycpy(b->ids,ids,len);CHKERRQ(ierr);
    }
  }
  ierr = PetscOptionsEnd();CHKERRQ(ierr);
  PetscFunctionReturn(0);
}
#endif

/* Behavior will be different for multi-physics or when using non-default boundary conditions */
static PetscErrorCode ModelSolutionSetDefault(Model mod,SolutionFunction func,void *ctx)
{
  PetscFunctionBeginUser;
  mod->solution    = func;
  mod->solutionctx = ctx;
  PetscFunctionReturn(0);
}

static PetscErrorCode ModelFunctionalRegister(Model mod,const char *name,PetscInt *offset,FunctionalFunction func,void *ctx)
{
  PetscErrorCode ierr;
  FunctionalLink link,*ptr;
  PetscInt       lastoffset = -1;

  PetscFunctionBeginUser;
  for (ptr=&mod->functionalRegistry; *ptr; ptr = &(*ptr)->next) lastoffset = (*ptr)->offset;
  ierr         = PetscNew(&link);CHKERRQ(ierr);
  ierr         = PetscStrallocpy(name,&link->name);CHKERRQ(ierr);
  link->offset = lastoffset + 1;
  link->func   = func;
  link->ctx    = ctx;
  link->next   = NULL;
  *ptr         = link;
  *offset      = link->offset;
  PetscFunctionReturn(0);
}

static PetscErrorCode ModelFunctionalSetFromOptions(Model mod,PetscOptions *PetscOptionsObject)
{
  PetscErrorCode ierr;
  PetscInt       i,j;
  FunctionalLink link;
  char           *names[256];

  PetscFunctionBeginUser;
  mod->numMonitored = ALEN(names);
  ierr = PetscOptionsStringArray("-monitor","list of functionals to monitor","",names,&mod->numMonitored,NULL);CHKERRQ(ierr);
  /* Create list of functionals that will be computed somehow */
  ierr = PetscMalloc1(mod->numMonitored,&mod->functionalMonitored);CHKERRQ(ierr);
  /* Create index of calls that we will have to make to compute these functionals (over-allocation in general). */
  ierr = PetscMalloc1(mod->numMonitored,&mod->functionalCall);CHKERRQ(ierr);
  mod->numCall = 0;
  for (i=0; i<mod->numMonitored; i++) {
    for (link=mod->functionalRegistry; link; link=link->next) {
      PetscBool match;
      ierr = PetscStrcasecmp(names[i],link->name,&match);CHKERRQ(ierr);
      if (match) break;
    }
    if (!link) SETERRQ1(mod->comm,PETSC_ERR_USER,"No known functional '%s'",names[i]);
    mod->functionalMonitored[i] = link;
    for (j=0; j<i; j++) {
      if (mod->functionalCall[j]->func == link->func && mod->functionalCall[j]->ctx == link->ctx) goto next_name;
    }
    mod->functionalCall[mod->numCall++] = link; /* Just points to the first link using the result. There may be more results. */
next_name:
    ierr = PetscFree(names[i]);CHKERRQ(ierr);
  }

  /* Find out the maximum index of any functional computed by a function we will be calling (even if we are not using it) */
  mod->maxComputed = -1;
  for (link=mod->functionalRegistry; link; link=link->next) {
    for (i=0; i<mod->numCall; i++) {
      FunctionalLink call = mod->functionalCall[i];
      if (link->func == call->func && link->ctx == call->ctx) {
        mod->maxComputed = PetscMax(mod->maxComputed,link->offset);
      }
    }
  }
  PetscFunctionReturn(0);
}

static PetscErrorCode FunctionalLinkDestroy(FunctionalLink *link)
{
  PetscErrorCode ierr;
  FunctionalLink l,next;

  PetscFunctionBeginUser;
  if (!link) PetscFunctionReturn(0);
  l     = *link;
  *link = NULL;
  for (; l; l=next) {
    next = l->next;
    ierr = PetscFree(l->name);CHKERRQ(ierr);
    ierr = PetscFree(l);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

PetscErrorCode SetInitialCondition(DM dm, Vec X, User user)
{
  DM                 plex, dmCell;
  Model              mod = user->model;
  Vec                cellgeom;
  const PetscScalar *cgeom;
  PetscScalar       *x;
  PetscInt           cStart, cEnd, c;
  PetscErrorCode     ierr;

  PetscFunctionBeginUser;
  ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
  ierr = DMPlexGetGeometryFVM(plex, NULL, &cellgeom, NULL);CHKERRQ(ierr);
  ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
  ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
  ierr = VecGetArrayRead(cellgeom, &cgeom);CHKERRQ(ierr);
  ierr = VecGetArray(X, &x);CHKERRQ(ierr);
  for (c = cStart; c < cEnd; ++c) {
    const PetscFVCellGeom *cg;
    PetscScalar    *xc;

    ierr = DMPlexPointLocalRead(dmCell,c,cgeom,&cg);CHKERRQ(ierr);
    ierr = DMPlexPointGlobalRef(dm,c,x,&xc);CHKERRQ(ierr);
    if (xc) {ierr = (*mod->solution)(mod,0.0,cg->centroid,xc,mod->solutionctx);CHKERRQ(ierr);}
  }
  ierr = VecRestoreArrayRead(cellgeom, &cgeom);CHKERRQ(ierr);
  ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
  ierr = DMDestroy(&plex);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

static PetscErrorCode OutputVTK(DM dm, const char *filename, PetscViewer *viewer)
{
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = PetscViewerCreate(PetscObjectComm((PetscObject)dm), viewer);CHKERRQ(ierr);
  ierr = PetscViewerSetType(*viewer, PETSCVIEWERVTK);CHKERRQ(ierr);
  ierr = PetscViewerFileSetName(*viewer, filename);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

static PetscErrorCode MonitorVTK(TS ts,PetscInt stepnum,PetscReal time,Vec X,void *ctx)
{
  User           user = (User)ctx;
  DM             dm, plex;
  PetscViewer    viewer;
  char           filename[PETSC_MAX_PATH_LEN],*ftable = NULL;
  PetscReal      xnorm;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = PetscObjectSetName((PetscObject) X, "solution");CHKERRQ(ierr);
  ierr = VecGetDM(X,&dm);CHKERRQ(ierr);
  ierr = VecNorm(X,NORM_INFINITY,&xnorm);CHKERRQ(ierr);
  if (stepnum >= 0) {           /* No summary for final time */
    Model             mod = user->model;
    Vec               cellgeom;
    PetscInt          c,cStart,cEnd,fcount,i;
    size_t            ftableused,ftablealloc;
    const PetscScalar *cgeom,*x;
    DM                dmCell;
    PetscReal         *fmin,*fmax,*fintegral,*ftmp;

    ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
    ierr = DMPlexGetGeometryFVM(plex, NULL, &cellgeom, NULL);CHKERRQ(ierr);
    fcount = mod->maxComputed+1;
    ierr   = PetscMalloc4(fcount,&fmin,fcount,&fmax,fcount,&fintegral,fcount,&ftmp);CHKERRQ(ierr);
    for (i=0; i<fcount; i++) {
      fmin[i]      = PETSC_MAX_REAL;
      fmax[i]      = PETSC_MIN_REAL;
      fintegral[i] = 0;
    }
    ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
    ierr = VecGetDM(cellgeom,&dmCell);CHKERRQ(ierr);
    ierr = VecGetArrayRead(cellgeom,&cgeom);CHKERRQ(ierr);
    ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
    for (c = cStart; c < cEnd; ++c) {
      const PetscFVCellGeom    *cg;
      const PetscScalar *cx;
      ierr = DMPlexPointLocalRead(dmCell,c,cgeom,&cg);CHKERRQ(ierr);
      ierr = DMPlexPointGlobalRead(dm,c,x,&cx);CHKERRQ(ierr);
      if (!cx) continue;        /* not a global cell */
      for (i=0; i<mod->numCall; i++) {
        FunctionalLink flink = mod->functionalCall[i];
        ierr = (*flink->func)(mod,time,cg->centroid,cx,ftmp,flink->ctx);CHKERRQ(ierr);
      }
      for (i=0; i<fcount; i++) {
        fmin[i]       = PetscMin(fmin[i],ftmp[i]);
        fmax[i]       = PetscMax(fmax[i],ftmp[i]);
        fintegral[i] += cg->volume * ftmp[i];
      }
    }
    ierr = VecRestoreArrayRead(cellgeom,&cgeom);CHKERRQ(ierr);
    ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
    ierr = DMDestroy(&plex);CHKERRQ(ierr);
    ierr = MPI_Allreduce(MPI_IN_PLACE,fmin,fcount,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)ts));CHKERRMPI(ierr);
    ierr = MPI_Allreduce(MPI_IN_PLACE,fmax,fcount,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)ts));CHKERRMPI(ierr);
    ierr = MPI_Allreduce(MPI_IN_PLACE,fintegral,fcount,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)ts));CHKERRMPI(ierr);

    ftablealloc = fcount * 100;
    ftableused  = 0;
    ierr        = PetscMalloc1(ftablealloc,&ftable);CHKERRQ(ierr);
    for (i=0; i<mod->numMonitored; i++) {
      size_t         countused;
      char           buffer[256],*p;
      FunctionalLink flink = mod->functionalMonitored[i];
      PetscInt       id    = flink->offset;
      if (i % 3) {
        ierr = PetscArraycpy(buffer,"  ",2);CHKERRQ(ierr);
        p    = buffer + 2;
      } else if (i) {
        char newline[] = "\n";
        ierr = PetscArraycpy(buffer,newline,sizeof(newline)-1);CHKERRQ(ierr);
        p    = buffer + sizeof(newline) - 1;
      } else {
        p = buffer;
      }
      ierr = PetscSNPrintfCount(p,sizeof buffer-(p-buffer),"%12s [%10.7g,%10.7g] int %10.7g",&countused,flink->name,(double)fmin[id],(double)fmax[id],(double)fintegral[id]);CHKERRQ(ierr);
      countused += p - buffer;
      if (countused > ftablealloc-ftableused-1) { /* reallocate */
        char *ftablenew;
        ftablealloc = 2*ftablealloc + countused;
        ierr = PetscMalloc(ftablealloc,&ftablenew);CHKERRQ(ierr);
        ierr = PetscArraycpy(ftablenew,ftable,ftableused);CHKERRQ(ierr);
        ierr = PetscFree(ftable);CHKERRQ(ierr);
        ftable = ftablenew;
      }
      ierr = PetscArraycpy(ftable+ftableused,buffer,countused);CHKERRQ(ierr);
      ftableused += countused;
      ftable[ftableused] = 0;
    }
    ierr = PetscFree4(fmin,fmax,fintegral,ftmp);CHKERRQ(ierr);

    ierr = PetscPrintf(PetscObjectComm((PetscObject)ts),"% 3D  time %8.4g  |x| %8.4g  %s\n",stepnum,(double)time,(double)xnorm,ftable ? ftable : "");CHKERRQ(ierr);
    ierr = PetscFree(ftable);CHKERRQ(ierr);
  }
  if (user->vtkInterval < 1) PetscFunctionReturn(0);
  if ((stepnum == -1) ^ (stepnum % user->vtkInterval == 0)) {
    if (stepnum == -1) {        /* Final time is not multiple of normal time interval, write it anyway */
      ierr = TSGetStepNumber(ts,&stepnum);CHKERRQ(ierr);
    }
    ierr = PetscSNPrintf(filename,sizeof filename,"ex11-%03D.vtu",stepnum);CHKERRQ(ierr);
    ierr = OutputVTK(dm,filename,&viewer);CHKERRQ(ierr);
    ierr = VecView(X,viewer);CHKERRQ(ierr);
    ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}

static PetscErrorCode OutputBIN(DM dm, const char *filename, PetscViewer *viewer)
{
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = PetscViewerCreate(PetscObjectComm((PetscObject)dm), viewer);CHKERRQ(ierr);
  ierr = PetscViewerSetType(*viewer, PETSCVIEWERBINARY);CHKERRQ(ierr);
  ierr = PetscViewerFileSetMode(*viewer,FILE_MODE_WRITE);CHKERRQ(ierr);
  ierr = PetscViewerFileSetName(*viewer, filename);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

static PetscErrorCode TestMonitor(DM dm, const char *filename, Vec X, PetscReal time)
{
  Vec            odesolution;
  PetscInt       Nr;
  PetscBool      equal;
  PetscReal      timeread;
  PetscViewer    viewer;
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
  ierr = VecCreate(PETSC_COMM_WORLD,&odesolution);CHKERRQ(ierr);
  ierr = VecLoad(odesolution,viewer);CHKERRQ(ierr);
  VecEqual(X,odesolution,&equal);
  if (!equal) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_FILE_UNEXPECTED,"Error in reading the vec data from file");
  else {
    ierr = PetscPrintf(PETSC_COMM_WORLD,"IO test OK for Vec\n");CHKERRQ(ierr);
  }
  /*Nr   = 1;
   ierr = PetscRealLoad(Nr,&Nr,&timeread,viewer);CHKERRQ(ierr);*/
  ierr = PetscViewerBinaryRead(viewer,&timeread,1,NULL,PETSC_REAL);CHKERRQ(ierr);

  if (timeread!=time) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_FILE_UNEXPECTED,"Error in reading the scalar data from file");
  else {
    ierr = PetscPrintf(PETSC_COMM_WORLD,"IO test OK for PetscReal\n");CHKERRQ(ierr);
  }

  ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
  ierr = VecDestroy(&odesolution);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

static PetscErrorCode MonitorBIN(TS ts,PetscInt stepnum,PetscReal time,Vec X,void *ctx)
{
  User           user = (User)ctx;
  DM             dm;
  PetscViewer    viewer;
  char           filename[PETSC_MAX_PATH_LEN];
  PetscErrorCode ierr;

  PetscFunctionBeginUser;
  ierr = VecGetDM(X,&dm);CHKERRQ(ierr);
  ierr = PetscSNPrintf(filename,sizeof filename,"ex11-SA-%06d.bin",stepnum);CHKERRQ(ierr);
  ierr = OutputBIN(dm,filename,&viewer);CHKERRQ(ierr);
  ierr = VecView(X,viewer);CHKERRQ(ierr);
  ierr = PetscRealView(1,&time,viewer);CHKERRQ(ierr);
  /* ierr = PetscViewerBinaryWrite(viewer,&time,1,PETSC_REAL);CHKERRQ(ierr);*/
  ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
  ierr = TestMonitor(dm,filename,X,time);CHKERRQ(ierr);
  PetscFunctionReturn(0);
}

int main(int argc, char **argv)
{
  MPI_Comm          comm;
  PetscFV           fvm;
  User              user;
  Model             mod;
  Physics           phys;
  DM                dm, plex;
  PetscReal         ftime, cfl, dt, minRadius;
  PetscInt          dim, nsteps;
  TS                ts;
  TSConvergedReason reason;
  Vec               X;
  PetscViewer       viewer;
  PetscMPIInt       rank;
  PetscBool         vtkCellGeom, splitFaces;
  PetscInt          overlap, f;
  char              filename[PETSC_MAX_PATH_LEN] = "sevenside.exo";
  PetscErrorCode    ierr;

  ierr = PetscInitialize(&argc, &argv, (char*) 0, help);if (ierr) return ierr;
  comm = PETSC_COMM_WORLD;
  ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);

  ierr = PetscNew(&user);CHKERRQ(ierr);
  ierr = PetscNew(&user->model);CHKERRQ(ierr);
  ierr = PetscNew(&user->model->physics);CHKERRQ(ierr);
  mod  = user->model;
  phys = mod->physics;
  mod->comm = comm;

  /* Register physical models to be available on the command line */
  ierr = PetscFunctionListAdd(&PhysicsList,"advect"          ,PhysicsCreate_Advect);CHKERRQ(ierr);
  ierr = PetscFunctionListAdd(&PhysicsList,"sw"              ,PhysicsCreate_SW);CHKERRQ(ierr);
  ierr = PetscFunctionListAdd(&PhysicsList,"euler"           ,PhysicsCreate_Euler);CHKERRQ(ierr);

  ierr = PetscOptionsBegin(comm,NULL,"Unstructured Finite Volume Mesh Options","");CHKERRQ(ierr);
  {
    cfl  = 0.9 * 4; /* default SSPRKS2 with s=5 stages is stable for CFL number s-1 */
    ierr = PetscOptionsReal("-ufv_cfl","CFL number per step","",cfl,&cfl,NULL);CHKERRQ(ierr);
    ierr = PetscOptionsString("-f","Exodus.II filename to read","",filename,filename,sizeof(filename),NULL);CHKERRQ(ierr);
    splitFaces = PETSC_FALSE;
    ierr = PetscOptionsBool("-ufv_split_faces","Split faces between cell sets","",splitFaces,&splitFaces,NULL);CHKERRQ(ierr);
    overlap = 1;
    ierr = PetscOptionsInt("-ufv_mesh_overlap","Number of cells to overlap partitions","",overlap,&overlap,NULL);CHKERRQ(ierr);
    user->vtkInterval = 1;
    ierr = PetscOptionsInt("-ufv_vtk_interval","VTK output interval (0 to disable)","",user->vtkInterval,&user->vtkInterval,NULL);CHKERRQ(ierr);
    vtkCellGeom = PETSC_FALSE;
    ierr = PetscOptionsBool("-ufv_vtk_cellgeom","Write cell geometry (for debugging)","",vtkCellGeom,&vtkCellGeom,NULL);CHKERRQ(ierr);
  }
  ierr = PetscOptionsEnd();CHKERRQ(ierr);
  ierr = DMPlexCreateExodusFromFile(comm, filename, PETSC_TRUE, &dm);CHKERRQ(ierr);
  ierr = DMViewFromOptions(dm, NULL, "-dm_view");CHKERRQ(ierr);
  ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);

  ierr = PetscOptionsBegin(comm,NULL,"Unstructured Finite Volume Physics Options","");CHKERRQ(ierr);
  {
    PetscDS          prob;
    PetscErrorCode (*physcreate)(PetscDS,Model,Physics);
    char             physname[256]  = "advect";

    ierr = DMCreateLabel(dm, "Face Sets");CHKERRQ(ierr);
    ierr = PetscOptionsFList("-physics","Physics module to solve","",PhysicsList,physname,physname,sizeof physname,NULL);CHKERRQ(ierr);
    ierr = PetscFunctionListFind(PhysicsList,physname,&physcreate);CHKERRQ(ierr);
    ierr = PetscMemzero(phys,sizeof(struct _n_Physics));CHKERRQ(ierr);
    ierr = DMGetDS(dm,&prob);CHKERRQ(ierr);
    ierr = (*physcreate)(prob,mod,phys);CHKERRQ(ierr);
    mod->maxspeed = phys->maxspeed;
    /* Count number of fields and dofs */
    for (phys->nfields=0,phys->dof=0; phys->field_desc[phys->nfields].name; phys->nfields++) phys->dof += phys->field_desc[phys->nfields].dof;

    if (mod->maxspeed <= 0) SETERRQ1(comm,PETSC_ERR_ARG_WRONGSTATE,"Physics '%s' did not set maxspeed",physname);
    if (phys->dof <= 0) SETERRQ1(comm,PETSC_ERR_ARG_WRONGSTATE,"Physics '%s' did not set dof",physname);
    ierr = ModelFunctionalSetFromOptions(mod,PetscOptionsObject);CHKERRQ(ierr);
  }
  ierr = PetscOptionsEnd();CHKERRQ(ierr);
  {
    DM dmDist;

    ierr = DMSetBasicAdjacency(dm, PETSC_TRUE, PETSC_FALSE);CHKERRQ(ierr);
    ierr = DMPlexDistribute(dm, overlap, NULL, &dmDist);CHKERRQ(ierr);
    if (dmDist) {
      ierr = DMDestroy(&dm);CHKERRQ(ierr);
      dm   = dmDist;
    }
  }
  ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
  {
    DM gdm;

    ierr = DMPlexConstructGhostCells(dm, NULL, NULL, &gdm);CHKERRQ(ierr);
    ierr = DMDestroy(&dm);CHKERRQ(ierr);
    dm   = gdm;
    ierr = DMViewFromOptions(dm, NULL, "-dm_view");CHKERRQ(ierr);
  }
  if (splitFaces) {ierr = ConstructCellBoundary(dm, user);CHKERRQ(ierr);}
  ierr = SplitFaces(&dm, "split faces", user);CHKERRQ(ierr);

  ierr = PetscFVCreate(comm, &fvm);CHKERRQ(ierr);
  ierr = PetscFVSetFromOptions(fvm);CHKERRQ(ierr);
  ierr = DMSetNumFields(dm, phys->nfields);CHKERRQ(ierr);
  for (f = 0; f < phys->nfields; ++f) {ierr = DMSetField(dm, f, (PetscObject) fvm);CHKERRQ(ierr);}
  ierr = PetscFVSetNumComponents(fvm, phys->dof);CHKERRQ(ierr);
  ierr = PetscFVSetSpatialDimension(fvm, dim);CHKERRQ(ierr);

  /* Set up DM with section describing local vector and configure local vector. */
  ierr = SetUpLocalSpace(dm, user);CHKERRQ(ierr);

  ierr = TSCreate(comm, &ts);CHKERRQ(ierr);
  ierr = TSSetType(ts, TSSSP);CHKERRQ(ierr);
  /* ierr = TSSetType(ts, TSRK);CHKERRQ(ierr); */
  ierr = TSSetDM(ts, dm);CHKERRQ(ierr);
  /* ierr = TSMonitorSet(ts,MonitorVTK,user,NULL);CHKERRQ(ierr); */
  ierr = TSMonitorSet(ts,MonitorBIN,user,NULL);CHKERRQ(ierr);
  ierr = DMTSSetRHSFunctionLocal(dm, DMPlexTSComputeRHSFunctionFVM, user);CHKERRQ(ierr);
  ierr = DMCreateGlobalVector(dm, &X);CHKERRQ(ierr);
  ierr = PetscObjectSetName((PetscObject) X, "solution");CHKERRQ(ierr);
  ierr = SetInitialCondition(dm, X, user);CHKERRQ(ierr);
  ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
  if (vtkCellGeom) {
    DM  dmCell;
    Vec cellgeom, partition;

    ierr = DMPlexGetGeometryFVM(plex, NULL, &cellgeom, NULL);CHKERRQ(ierr);
    ierr = OutputVTK(dm, "ex11-cellgeom.vtk", &viewer);CHKERRQ(ierr);
    ierr = VecView(cellgeom, viewer);CHKERRQ(ierr);
    ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
    ierr = CreatePartitionVec(dm, &dmCell, &partition);CHKERRQ(ierr);
    ierr = OutputVTK(dmCell, "ex11-partition.vtk", &viewer);CHKERRQ(ierr);
    ierr = VecView(partition, viewer);CHKERRQ(ierr);
    ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
    ierr = VecDestroy(&partition);CHKERRQ(ierr);
    ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
  }

  ierr = DMPlexGetGeometryFVM(plex, NULL, NULL, &minRadius);CHKERRQ(ierr);
  ierr = DMDestroy(&plex);CHKERRQ(ierr);
  ierr = TSSetMaxTime(ts,2.0);CHKERRQ(ierr);
  ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);CHKERRQ(ierr);
  dt   = cfl * minRadius / user->model->maxspeed;
  ierr = TSSetTimeStep(ts,dt);CHKERRQ(ierr);
  ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
  ierr = TSSolve(ts,X);CHKERRQ(ierr);
  ierr = TSGetSolveTime(ts,&ftime);CHKERRQ(ierr);
  ierr = TSGetStepNumber(ts,&nsteps);CHKERRQ(ierr);
  ierr = TSGetConvergedReason(ts,&reason);CHKERRQ(ierr);
  ierr = PetscPrintf(PETSC_COMM_WORLD,"%s at time %g after %D steps\n",TSConvergedReasons[reason],(double)ftime,nsteps);CHKERRQ(ierr);
  ierr = TSDestroy(&ts);CHKERRQ(ierr);

  ierr = PetscFunctionListDestroy(&PhysicsList);CHKERRQ(ierr);
  ierr = FunctionalLinkDestroy(&user->model->functionalRegistry);CHKERRQ(ierr);
  ierr = PetscFree(user->model->functionalMonitored);CHKERRQ(ierr);
  ierr = PetscFree(user->model->functionalCall);CHKERRQ(ierr);
  ierr = PetscFree(user->model->physics->data);CHKERRQ(ierr);
  ierr = PetscFree(user->model->physics);CHKERRQ(ierr);
  ierr = PetscFree(user->model);CHKERRQ(ierr);
  ierr = PetscFree(user);CHKERRQ(ierr);
  ierr = VecDestroy(&X);CHKERRQ(ierr);
  ierr = PetscFVDestroy(&fvm);CHKERRQ(ierr);
  ierr = DMDestroy(&dm);CHKERRQ(ierr);
  ierr = PetscFinalize();
  return ierr;
}
